package net.bwie.realtime.jtp.dwd.log.job;

import com.alibaba.fastjson.JSON;
import net.bwie.realtime.jtp.dwd.log.function.AdjustIsNewProcessFunction;
import net.bwie.realtime.jtp.dwd.log.function.LogSplitProcessFunction;
import net.bwie.realtime.jtp.common.utils.KafkaUtil;
import net.bwie.realtime.jtp.dwd.log.function.jtpLogClickHouse;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

public class JtpLogEtlJob {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        DataStream<String> kafkaDataStream = KafkaUtil.consumerKafka(env, "topic-log");
        // kafkaDataStream.print("kafka");
        
        // 3.数据转换-transformation
        // 4.数据输出-sink
        processLog(kafkaDataStream) ;

        env.execute("JtpLogEtlJob");
    }

    private static void processLog(DataStream<String> logStream) {
        // 数据清洗
        DataStream<String> cleanedStream = logCleaned(logStream);
        // 新老客户状态修复
        DataStream<String> isNewStream = processIsNew(cleanedStream);
        // 数据分流
        DataStream<String> pageStream = splitStream(isNewStream);
        // 存储页面明细表
//        jtpLogClickHouse.savePageLog(pageStream);
        // 存储页面数据
//        KafkaUtil.producerKafka(pageStream,"dwd-traffic-page-log");
     }

    private static DataStream<String> splitStream(DataStream<String> isNewStream) {

        // 定义侧流Tag
        OutputTag<String> errTag = new OutputTag<String>("errTag") {};
        OutputTag<String> startTag = new OutputTag<String>("startTag") {};
        OutputTag<String> displayTag = new OutputTag<String>("displayTag") {};
        OutputTag<String> actionTag = new OutputTag<String>("actionTag") {};

        SingleOutputStreamOperator<String> pageStream = isNewStream.process(
                //调用方法传入形参
                new LogSplitProcessFunction(errTag,startTag,displayTag,actionTag)
        );
        // 获取所有侧流
        DataStream<String> errStream = pageStream.getSideOutput(errTag);
        DataStream<String> startStream = pageStream.getSideOutput(startTag);
        DataStream<String> displayStream = pageStream.getSideOutput(displayTag);
        DataStream<String> actionStream = pageStream.getSideOutput(actionTag);

        jtpLogClickHouse.saveActionLog(actionStream);
//        jtpLogClickHouse.saveStartLog(startStream);

        // 侧流写入topic
//        KafkaUtil.producerKafka(errStream,"dwd-traffic-err-log");
//        KafkaUtil.producerKafka(startStream,"dwd-traffic-start-log");
//        KafkaUtil.producerKafka(displayStream,"dwd-traffic-display-log");
//        KafkaUtil.producerKafka(actionStream,"dwd-traffic-action-log");
        // 返回页面主流
        return pageStream;
    }

    private static DataStream<String> processIsNew(DataStream<String> cleanedStream) {
        KeyedStream<String, String> midStream = cleanedStream.keyBy(
                t -> JSON.parseObject(t).getJSONObject("common").getString("mid")
        );
        return midStream.process(new AdjustIsNewProcessFunction());
    }

    private static DataStream<String> logCleaned(DataStream<String> logStream) {

        OutputTag<String> dirtyTag = new OutputTag<String>("dirtyTag") {};

        SingleOutputStreamOperator<String> cleanedStream = logStream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String s, Context context, Collector<String> collector) throws Exception {
                try {
                    // 解析JSON数据
                    JSON.parseObject(s);
                    // 没有异常收集主流
                    collector.collect(s);
                } catch (Exception e) {
                    // 异常存入异常topic
                    context.output(dirtyTag,s);
                }
            }
        });
        DataStream<String> dirtyStream = cleanedStream.getSideOutput(dirtyTag);
        KafkaUtil.producerKafka(dirtyStream,"dwd-traffic-dirty-log");
        return cleanedStream;
    }

}